{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "9d35cc5e",
   "metadata": {
    "cell_id": 39
   },
   "source": [
    "### 加载数据集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "45a6bee8",
   "metadata": {
    "cell_id": 1
   },
   "outputs": [],
   "source": [
    "from utils import load_corpus_bert\n",
    "import numpy as np\n",
    "\n",
    "TRAIN_PATH = \"./data/weibo2018/train.txt\"\n",
    "TEST_PATH = \"./data/weibo2018/test.txt\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "8369026b",
   "metadata": {
    "cell_id": 3
   },
   "outputs": [],
   "source": [
    "# 分别加载训练集和测试集\n",
    "train_data = load_corpus_bert(TRAIN_PATH)\n",
    "test_data = load_corpus_bert(TEST_PATH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "725a34f0",
   "metadata": {
    "cell_id": 4
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>“书中自有黄金屋，书中自有颜如玉”。沿着岁月的长河跋涉，或是风光旖旎，或是姹紫嫣红，万千...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>这是英超被黑的最惨的一次[二哈][二哈]十几年来，中国只有孙继海，董方卓，郑智，李铁登陆过英...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>中国远洋海运集团副总经理俞曾港4月21日在 上表示，中央企业“走出去”是要站在更高的平台参...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>看《流星花园》其实也还好啦，现在的观念以及时尚眼光都不一样了，或许十几年之后的人看我们的现在...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>汉武帝的罪己诏的真实性尽管存在着争议，然而“轮台罪己诏”作为中国历史上第一份皇帝自我批评的文...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9995</th>\n",
       "      <td>火车上碰见这种占别人位置还理直气壮的王八蛋真的心累，报了乘警半天不见人，只好祝病魔早日战胜之...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9996</th>\n",
       "      <td>倒霉催的，坐上晚点一个多小时的汽车🚗，在高速上司机叔叔说他没听清我说话，so我一路超想上厕所...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9997</th>\n",
       "      <td>急诊第一天上班，说不上的心累，这漫长的两个月如何过啊[悲伤][悲伤][悲伤]  \\n</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9998</th>\n",
       "      <td>我每个月供着爱奇艺，网易云，快连，芒果TV，包图网，这些都是大企业啊，我也是心累😂😂😂😱😱😱...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9999</th>\n",
       "      <td>其实这种两家毫无交集 因为一个粉丝单方面撩骚就撕成一片上升成两家战火的情况真的很煤意思 要是...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10000 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   text  label\n",
       "0       “书中自有黄金屋，书中自有颜如玉”。沿着岁月的长河跋涉，或是风光旖旎，或是姹紫嫣红，万千...      1\n",
       "1     这是英超被黑的最惨的一次[二哈][二哈]十几年来，中国只有孙继海，董方卓，郑智，李铁登陆过英...      0\n",
       "2      中国远洋海运集团副总经理俞曾港4月21日在 上表示，中央企业“走出去”是要站在更高的平台参...      1\n",
       "3     看《流星花园》其实也还好啦，现在的观念以及时尚眼光都不一样了，或许十几年之后的人看我们的现在...      1\n",
       "4     汉武帝的罪己诏的真实性尽管存在着争议，然而“轮台罪己诏”作为中国历史上第一份皇帝自我批评的文...      1\n",
       "...                                                 ...    ...\n",
       "9995  火车上碰见这种占别人位置还理直气壮的王八蛋真的心累，报了乘警半天不见人，只好祝病魔早日战胜之...      0\n",
       "9996  倒霉催的，坐上晚点一个多小时的汽车🚗，在高速上司机叔叔说他没听清我说话，so我一路超想上厕所...      0\n",
       "9997         急诊第一天上班，说不上的心累，这漫长的两个月如何过啊[悲伤][悲伤][悲伤]  \\n      0\n",
       "9998  我每个月供着爱奇艺，网易云，快连，芒果TV，包图网，这些都是大企业啊，我也是心累😂😂😂😱😱😱...      0\n",
       "9999  其实这种两家毫无交集 因为一个粉丝单方面撩骚就撕成一片上升成两家战火的情况真的很煤意思 要是...      0\n",
       "\n",
       "[10000 rows x 2 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "df_train = pd.DataFrame(train_data, columns=[\"text\", \"label\"])\n",
    "df_test = pd.DataFrame(test_data, columns=[\"text\", \"label\"])\n",
    "df_train\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "07db7f2c",
   "metadata": {
    "cell_id": 41
   },
   "source": [
    "### 加载Bert"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "67761c0d",
   "metadata": {
    "cell_id": 5
   },
   "outputs": [],
   "source": [
    "import os\n",
    "from transformers import BertTokenizer, BertModel\n",
    "\n",
    "os.environ[\"KMP_DUPLICATE_LIB_OK\"] = \"TRUE\"\n",
    "MODEL_PATH = \"./model/chinese_wwm_pytorch\"     # 下载地址见 https://github.com/ymcui/Chinese-BERT-wwm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "a7e96c02",
   "metadata": {
    "cell_id": 6
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at ./model/chinese_wwm_pytorch were not used when initializing BertModel: ['cls.predictions.transform.dense.weight', 'cls.seq_relationship.bias', 'cls.seq_relationship.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.decoder.weight']\n",
      "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    }
   ],
   "source": [
    "# 加载\n",
    "tokenizer = BertTokenizer.from_pretrained(MODEL_PATH)   # 分词器\n",
    "bert = BertModel.from_pretrained(MODEL_PATH)            # 模型"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4da4f205",
   "metadata": {
    "cell_id": 43
   },
   "source": [
    "### 神经网络"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "fedd1dcc",
   "metadata": {
    "cell_id": 7
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch import nn\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "\n",
    "device = \"cuda:0\" if torch.cuda.is_available() else \"cpu\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "7c83f073",
   "metadata": {
    "cell_id": 8
   },
   "outputs": [],
   "source": [
    "# 超参数\n",
    "learning_rate = 1e-3\n",
    "input_size = 768\n",
    "num_epoches = 10\n",
    "batch_size = 100\n",
    "decay_rate = 0.9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "ad8a11e8",
   "metadata": {
    "cell_id": 9
   },
   "outputs": [],
   "source": [
    "# 数据集\n",
    "class MyDataset(Dataset):\n",
    "    def __init__(self, df):\n",
    "        self.data = df[\"text\"].tolist()\n",
    "        self.label = df[\"label\"].tolist()\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        data = self.data[index]\n",
    "        label = self.label[index]\n",
    "        return data, label\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.label)\n",
    "\n",
    "# 训练集\n",
    "train_data = MyDataset(df_train)\n",
    "train_loader = DataLoader(train_data, batch_size=batch_size, shuffle=True)\n",
    "\n",
    "# 测试集\n",
    "test_data = MyDataset(df_test)\n",
    "test_loader = DataLoader(test_data, batch_size=batch_size, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "69602a96",
   "metadata": {
    "cell_id": 10
   },
   "outputs": [],
   "source": [
    "# 网络结构\n",
    "class Net(nn.Module):\n",
    "    def __init__(self, input_size):\n",
    "        super(Net, self).__init__()\n",
    "        self.fc = nn.Linear(input_size, 1)\n",
    "        self.sigmoid = nn.Sigmoid()\n",
    "\n",
    "    def forward(self, x):\n",
    "        out = self.fc(x)\n",
    "        out = self.sigmoid(out)\n",
    "        return out\n",
    "\n",
    "net = Net(input_size).to(device)\n",
    "bert = bert.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "5f90ef00",
   "metadata": {
    "cell_id": 34
   },
   "outputs": [],
   "source": [
    "from sklearn import metrics\n",
    "\n",
    "# 测试集效果检验\n",
    "def test():\n",
    "    y_pred, y_true = [], []\n",
    "\n",
    "    with torch.no_grad():\n",
    "        for words, labels in test_loader:\n",
    "            tokens = tokenizer(words, padding=True)\n",
    "            input_ids = torch.tensor(tokens[\"input_ids\"]).to(device)\n",
    "            attention_mask = torch.tensor(tokens[\"attention_mask\"]).to(device)\n",
    "            last_hidden_states = bert(input_ids, attention_mask=attention_mask)\n",
    "            bert_output = last_hidden_states[0][:, 0]\n",
    "            outputs = net(bert_output)          # 前向传播\n",
    "            outputs = outputs.view(-1)          # 将输出展平\n",
    "            y_pred.append(outputs)\n",
    "            y_true.append(labels)\n",
    "\n",
    "    y_prob = torch.cat(y_pred)\n",
    "    y_true = torch.cat(y_true)\n",
    "    y_pred = y_prob.clone()\n",
    "    y_pred[y_pred > 0.5] = 1\n",
    "    y_pred[y_pred <= 0.5] = 0\n",
    "    \n",
    "    y_true = y_true.cpu()\n",
    "    y_pred = y_pred.cpu()\n",
    "    y_prob = y_prob.cpu()\n",
    "    print(metrics.classification_report(y_true, y_pred))\n",
    "    print(\"准确率:\", metrics.accuracy_score(y_true, y_pred))\n",
    "    print(\"AUC:\", metrics.roc_auc_score(y_true, y_prob))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "7fb295f9",
   "metadata": {
    "cell_id": 11
   },
   "outputs": [],
   "source": [
    "# 定义损失函数和优化器\n",
    "criterion = nn.BCELoss()\n",
    "optimizer = torch.optim.Adam(net.parameters(), lr=learning_rate)\n",
    "scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=decay_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "e483cb2c",
   "metadata": {
    "cell_id": 14,
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch:1, step:10, loss:0.6813191771507263\n",
      "epoch:1, step:20, loss:0.6367926001548767\n",
      "epoch:1, step:30, loss:0.5908938646316528\n",
      "epoch:1, step:40, loss:0.5482066869735718\n",
      "epoch:1, step:50, loss:0.5113250017166138\n",
      "epoch:1, step:60, loss:0.5121451616287231\n",
      "epoch:1, step:70, loss:0.5019151568412781\n",
      "epoch:1, step:80, loss:0.5041934251785278\n",
      "epoch:1, step:90, loss:0.49353915452957153\n",
      "epoch:1, step:100, loss:0.4792908728122711\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.72      0.79      0.75       155\n",
      "           1       0.90      0.86      0.88       345\n",
      "\n",
      "    accuracy                           0.84       500\n",
      "   macro avg       0.81      0.83      0.82       500\n",
      "weighted avg       0.85      0.84      0.84       500\n",
      "\n",
      "准确率: 0.84\n",
      "AUC: 0.9012435717625058\n",
      "saved model:  ./model/bert_dnn_1.model\n",
      "epoch:2, step:10, loss:0.46031346917152405\n",
      "epoch:2, step:20, loss:0.46189332008361816\n",
      "epoch:2, step:30, loss:0.4660502076148987\n",
      "epoch:2, step:40, loss:0.4732876420021057\n",
      "epoch:2, step:50, loss:0.4567083418369293\n",
      "epoch:2, step:60, loss:0.45447808504104614\n",
      "epoch:2, step:70, loss:0.4462393820285797\n",
      "epoch:2, step:80, loss:0.4350458085536957\n",
      "epoch:2, step:90, loss:0.45661574602127075\n",
      "epoch:2, step:100, loss:0.43131086230278015\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.72      0.83      0.77       155\n",
      "           1       0.92      0.85      0.88       345\n",
      "\n",
      "    accuracy                           0.85       500\n",
      "   macro avg       0.82      0.84      0.83       500\n",
      "weighted avg       0.86      0.85      0.85       500\n",
      "\n",
      "准确率: 0.846\n",
      "AUC: 0.9128938756428238\n",
      "saved model:  ./model/bert_dnn_2.model\n",
      "epoch:3, step:10, loss:0.432222843170166\n",
      "epoch:3, step:20, loss:0.43521466851234436\n",
      "epoch:3, step:30, loss:0.4310072064399719\n",
      "epoch:3, step:40, loss:0.41495609283447266\n",
      "epoch:3, step:50, loss:0.4109768867492676\n",
      "epoch:3, step:60, loss:0.45129290223121643\n",
      "epoch:3, step:70, loss:0.43410035967826843\n",
      "epoch:3, step:80, loss:0.4282527565956116\n",
      "epoch:3, step:90, loss:0.43675899505615234\n",
      "epoch:3, step:100, loss:0.44812995195388794\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.72      0.84      0.77       155\n",
      "           1       0.92      0.85      0.89       345\n",
      "\n",
      "    accuracy                           0.85       500\n",
      "   macro avg       0.82      0.85      0.83       500\n",
      "weighted avg       0.86      0.85      0.85       500\n",
      "\n",
      "准确率: 0.848\n",
      "AUC: 0.9171575502571295\n",
      "saved model:  ./model/bert_dnn_3.model\n",
      "epoch:4, step:10, loss:0.43588048219680786\n",
      "epoch:4, step:20, loss:0.41449791193008423\n",
      "epoch:4, step:30, loss:0.42159825563430786\n",
      "epoch:4, step:40, loss:0.43740397691726685\n",
      "epoch:4, step:50, loss:0.4103790819644928\n",
      "epoch:4, step:60, loss:0.4272201955318451\n",
      "epoch:4, step:70, loss:0.401478111743927\n",
      "epoch:4, step:80, loss:0.428305447101593\n",
      "epoch:4, step:90, loss:0.4094158709049225\n",
      "epoch:4, step:100, loss:0.4253961741924286\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.75      0.81      0.78       155\n",
      "           1       0.91      0.88      0.90       345\n",
      "\n",
      "    accuracy                           0.86       500\n",
      "   macro avg       0.83      0.85      0.84       500\n",
      "weighted avg       0.86      0.86      0.86       500\n",
      "\n",
      "准确率: 0.858\n",
      "AUC: 0.9216830294530154\n",
      "saved model:  ./model/bert_dnn_4.model\n",
      "epoch:5, step:10, loss:0.39831170439720154\n",
      "epoch:5, step:20, loss:0.41463977098464966\n",
      "epoch:5, step:30, loss:0.41155534982681274\n",
      "epoch:5, step:40, loss:0.4231754243373871\n",
      "epoch:5, step:50, loss:0.4058705270290375\n",
      "epoch:5, step:60, loss:0.41910943388938904\n",
      "epoch:5, step:70, loss:0.41561374068260193\n",
      "epoch:5, step:80, loss:0.4222615659236908\n",
      "epoch:5, step:90, loss:0.3959978520870209\n",
      "epoch:5, step:100, loss:0.4395703375339508\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.75      0.81      0.78       155\n",
      "           1       0.91      0.88      0.90       345\n",
      "\n",
      "    accuracy                           0.86       500\n",
      "   macro avg       0.83      0.85      0.84       500\n",
      "weighted avg       0.86      0.86      0.86       500\n",
      "\n",
      "准确率: 0.86\n",
      "AUC: 0.9242262739597944\n",
      "saved model:  ./model/bert_dnn_5.model\n",
      "epoch:6, step:10, loss:0.3959680199623108\n",
      "epoch:6, step:20, loss:0.4046095907688141\n",
      "epoch:6, step:30, loss:0.40109163522720337\n",
      "epoch:6, step:40, loss:0.38858869671821594\n",
      "epoch:6, step:50, loss:0.41756463050842285\n",
      "epoch:6, step:60, loss:0.46486639976501465\n",
      "epoch:6, step:70, loss:0.41850075125694275\n",
      "epoch:6, step:80, loss:0.39176854491233826\n",
      "epoch:6, step:90, loss:0.3868580758571625\n",
      "epoch:6, step:100, loss:0.43457213044166565\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.77      0.81      0.79       155\n",
      "           1       0.91      0.89      0.90       345\n",
      "\n",
      "    accuracy                           0.87       500\n",
      "   macro avg       0.84      0.85      0.85       500\n",
      "weighted avg       0.87      0.87      0.87       500\n",
      "\n",
      "准确率: 0.866\n",
      "AUC: 0.9268069191210846\n",
      "saved model:  ./model/bert_dnn_6.model\n",
      "epoch:7, step:10, loss:0.4059884250164032\n",
      "epoch:7, step:20, loss:0.40888792276382446\n",
      "epoch:7, step:30, loss:0.39916667342185974\n",
      "epoch:7, step:40, loss:0.4073812663555145\n",
      "epoch:7, step:50, loss:0.4053972363471985\n",
      "epoch:7, step:60, loss:0.4051210582256317\n",
      "epoch:7, step:70, loss:0.41913318634033203\n",
      "epoch:7, step:80, loss:0.4051087498664856\n",
      "epoch:7, step:90, loss:0.41990962624549866\n",
      "epoch:7, step:100, loss:0.39672717452049255\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.77      0.81      0.79       155\n",
      "           1       0.91      0.89      0.90       345\n",
      "\n",
      "    accuracy                           0.86       500\n",
      "   macro avg       0.84      0.85      0.84       500\n",
      "weighted avg       0.87      0.86      0.86       500\n",
      "\n",
      "准确率: 0.864\n",
      "AUC: 0.9280598410472184\n",
      "saved model:  ./model/bert_dnn_7.model\n",
      "epoch:8, step:10, loss:0.3849932551383972\n",
      "epoch:8, step:20, loss:0.4146333634853363\n",
      "epoch:8, step:30, loss:0.40953317284584045\n",
      "epoch:8, step:40, loss:0.398123174905777\n",
      "epoch:8, step:50, loss:0.39869821071624756\n",
      "epoch:8, step:60, loss:0.38304370641708374\n",
      "epoch:8, step:70, loss:0.4102857708930969\n",
      "epoch:8, step:80, loss:0.4156968295574188\n",
      "epoch:8, step:90, loss:0.3973374366760254\n",
      "epoch:8, step:100, loss:0.42341551184654236\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.74      0.82      0.78       155\n",
      "           1       0.91      0.87      0.89       345\n",
      "\n",
      "    accuracy                           0.86       500\n",
      "   macro avg       0.83      0.85      0.84       500\n",
      "weighted avg       0.86      0.86      0.86       500\n",
      "\n",
      "准确率: 0.856\n",
      "AUC: 0.9291257597007947\n",
      "saved model:  ./model/bert_dnn_8.model\n",
      "epoch:9, step:10, loss:0.37348756194114685\n",
      "epoch:9, step:20, loss:0.421467125415802\n",
      "epoch:9, step:30, loss:0.3696461319923401\n",
      "epoch:9, step:40, loss:0.4180992543697357\n",
      "epoch:9, step:50, loss:0.41242972016334534\n",
      "epoch:9, step:60, loss:0.40393567085266113\n",
      "epoch:9, step:70, loss:0.3996097147464752\n",
      "epoch:9, step:80, loss:0.3827632665634155\n",
      "epoch:9, step:90, loss:0.4397268295288086\n",
      "epoch:9, step:100, loss:0.3957260549068451\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.75      0.82      0.78       155\n",
      "           1       0.92      0.88      0.89       345\n",
      "\n",
      "    accuracy                           0.86       500\n",
      "   macro avg       0.83      0.85      0.84       500\n",
      "weighted avg       0.86      0.86      0.86       500\n",
      "\n",
      "准确率: 0.858\n",
      "AUC: 0.929256661991585\n",
      "saved model:  ./model/bert_dnn_9.model\n",
      "epoch:10, step:10, loss:0.38457533717155457\n",
      "epoch:10, step:20, loss:0.4068971276283264\n",
      "epoch:10, step:30, loss:0.40848761796951294\n",
      "epoch:10, step:40, loss:0.4041728675365448\n",
      "epoch:10, step:50, loss:0.4117070138454437\n",
      "epoch:10, step:60, loss:0.38697195053100586\n",
      "epoch:10, step:70, loss:0.4062800109386444\n",
      "epoch:10, step:80, loss:0.3946671485900879\n",
      "epoch:10, step:90, loss:0.40975210070610046\n",
      "epoch:10, step:100, loss:0.38564714789390564\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.76      0.82      0.79       155\n",
      "           1       0.92      0.88      0.90       345\n",
      "\n",
      "    accuracy                           0.86       500\n",
      "   macro avg       0.84      0.85      0.84       500\n",
      "weighted avg       0.87      0.86      0.86       500\n",
      "\n",
      "准确率: 0.862\n",
      "AUC: 0.9301168770453484\n",
      "saved model:  ./model/bert_dnn_10.model\n"
     ]
    }
   ],
   "source": [
    "# 迭代训练\n",
    "for epoch in range(num_epoches):\n",
    "    total_loss = 0\n",
    "    for i, (words, labels) in enumerate(train_loader):\n",
    "        tokens = tokenizer(words, padding=True)\n",
    "        input_ids = torch.tensor(tokens[\"input_ids\"]).to(device)\n",
    "        attention_mask = torch.tensor(tokens[\"attention_mask\"]).to(device)\n",
    "        labels = labels.float().to(device)\n",
    "        with torch.no_grad():\n",
    "            last_hidden_states = bert(input_ids, attention_mask=attention_mask)\n",
    "            bert_output = last_hidden_states[0][:, 0]\n",
    "        optimizer.zero_grad()               # 梯度清零\n",
    "        outputs = net(bert_output)          # 前向传播\n",
    "        logits = outputs.view(-1)           # 将输出展平\n",
    "        loss = criterion(logits, labels)    # loss计算\n",
    "        total_loss += loss\n",
    "        loss.backward()                     # 反向传播，计算梯度\n",
    "        optimizer.step()                    # 梯度更新\n",
    "        if (i+1) % 10 == 0:\n",
    "            print(\"epoch:{}, step:{}, loss:{}\".format(epoch+1, i+1, total_loss/10))\n",
    "            total_loss = 0\n",
    "    \n",
    "    # learning_rate decay\n",
    "    scheduler.step()\n",
    "    \n",
    "    # test\n",
    "    test()\n",
    "    \n",
    "    # save model\n",
    "    model_path = \"./model/bert_dnn_{}.model\".format(epoch+1)\n",
    "    torch.save(net, model_path)\n",
    "    print(\"saved model: \", model_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0496e25e",
   "metadata": {
    "cell_id": 23
   },
   "source": [
    "### 手动输入句子，判断情感倾向（1正/0负）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "ca0aeda8",
   "metadata": {
    "cell_id": 38
   },
   "outputs": [],
   "source": [
    "net = torch.load(\"./model/bert_dnn_8.model\")    # 训练过程中的巅峰时刻"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "d401769c",
   "metadata": {
    "cell_id": 37
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.8641],\n",
       "        [0.1763]], device='cuda:0', grad_fn=<SigmoidBackward0>)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s = [\"华丽繁荣的城市、充满回忆的小镇、郁郁葱葱的山谷...\", \"突然就觉得人间不值得\"]\n",
    "tokens = tokenizer(s, padding=True)\n",
    "input_ids = torch.tensor(tokens[\"input_ids\"]).to(device)\n",
    "attention_mask = torch.tensor(tokens[\"attention_mask\"]).to(device)\n",
    "last_hidden_states = bert(input_ids, attention_mask=attention_mask)\n",
    "bert_output = last_hidden_states[0][:, 0]\n",
    "outputs = net(bert_output)\n",
    "outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "a561f7dc",
   "metadata": {
    "cell_id": 27,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.9646],\n",
       "        [0.9848]], device='cuda:0', grad_fn=<SigmoidBackward0>)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s = [\"今天天气真好\", \"今天天气特别特别棒\"]\n",
    "tokens = tokenizer(s, padding=True)\n",
    "input_ids = torch.tensor(tokens[\"input_ids\"]).to(device)\n",
    "attention_mask = torch.tensor(tokens[\"attention_mask\"]).to(device)\n",
    "last_hidden_states = bert(input_ids, attention_mask=attention_mask)\n",
    "bert_output = last_hidden_states[0][:, 0]\n",
    "outputs = net(bert_output)\n",
    "outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "5d172b0d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.2533],\n",
       "        [0.6691]], device='cuda:0', grad_fn=<SigmoidBackward0>)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s = [\"外面在下雨\", \"我想吃牛肉\"]\n",
    "tokens = tokenizer(s, padding=True)\n",
    "input_ids = torch.tensor(tokens[\"input_ids\"]).to(device)\n",
    "attention_mask = torch.tensor(tokens[\"attention_mask\"]).to(device)\n",
    "last_hidden_states = bert(input_ids, attention_mask=attention_mask)\n",
    "bert_output = last_hidden_states[0][:, 0]\n",
    "outputs = net(bert_output)\n",
    "outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2b187c19",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  },
  "max_cell_id": 45,
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {},
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
